package com.atguigu.sparkcore.rdd.tranformsinglevale

import com.atguigu.sparkcore.util.MySparkContextUtil
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * rdd转换-Distinct
  * 去重
  */
object Distinct {

  def main(args: Array[String]): Unit = {

    // 获取sparkContext
    val sc = MySparkContextUtil.get(args)
    // 创建rdd
    //    val list: Array[Int] = Array(10, 20, 30, 40, 50, 10, 20)
    val list = List(new User(10, "10"), new User(10, "20"), new User(20, "20"))
    val rdd = sc.makeRDD(list, 2)

    // 转换rdd
    val result = rdd.distinct

    // 输出
    result.collect().foreach(println)

    // 关闭资源
    MySparkContextUtil.close(sc)
  }

  case class User(val age: Int, val name: String) extends Serializable {
    override def hashCode(): Int = this.age

    override def equals(obj: scala.Any): Boolean = {
      if (obj.isInstanceOf[User]) {
        val u = obj.asInstanceOf[User]
        u.age == this.age
      } else false
    }
  }

}


